package com.webmagic.house.scheduler;

import com.alibaba.fastjson.JSON;
import org.apache.commons.codec.digest.DigestUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
import us.codecraft.webmagic.*;
import us.codecraft.webmagic.scheduler.RedisScheduler;
import us.codecraft.webmagic.utils.HttpConstant;
import us.codecraft.webmagic.utils.UrlUtils;

import java.net.URI;
import java.net.URISyntaxException;
import java.util.Set;

/**
 * @author wen
 * @Title: ProjectScheduler
 * @ProjectName house
 * @Description: TODO
 * @date 2020/1/615:37
 */
public class ProjectScheduler extends RedisScheduler implements SpiderListener {
    private static Logger logger = LoggerFactory.getLogger(ProjectScheduler.class);
    private static final String ERR_SET_PREFIX = "error_set_";
    private static final String ITEM_PREFIX = "item_";
    private static final String QUEUE_PREFIX = "queue_";
    private String skipUrl;
    private  Site site;
    private String enableRepeat;
    public ProjectScheduler(String host, String skipUrl, Site site,String enableRepeat) {
        super(host);
        this.skipUrl = skipUrl;
        this.site = site;
        this.enableRepeat = enableRepeat;


    }

//    @Override
//    protected boolean noNeedToRemoveDuplicate(Request request) {
//        //return super.noNeedToRemoveDuplicate(request);
//        return HttpConstant.Method.POST.equalsIgnoreCase(request.getMethod());
//        //return false;
//    }

    @Override
    public boolean isDuplicate(Request request, Task task) {
        Jedis jedis = pool.getResource();
        try {
            if("yes".equals(enableRepeat)){
                return  false;
            }
            boolean isExist = jedis.sadd(getSetKey(task), request.getUrl()) == 0;
//            if(skipUrl!=null&& request.getUrl().matches(skipUrl)){
//                return false;
//            }
            return isExist;
            //return jedis.sadd(getSetKey(task), request.getUrl()) == 0;
        } finally {
            pool.returnResource(jedis);
        }
    }

    public void addLastFailTasks(Spider spider){
        Jedis jedis = pool.getResource();
        String domain = spider.getSite().getDomain();
        String key = ERR_SET_PREFIX+domain;
        Set<String> errStrings =  jedis.smembers(key);
        for (String url:errStrings){
            jedis.rpush(QUEUE_PREFIX+domain, url);
            //spider.addUrl()
            jedis.srem(key,url);
        }


    }


    @Override
    public void onSuccess(Request request) {

    }

    @Override
    public void onError(Request request) {
        Object cycleTriedTimesObject = request.getExtra(Request.CYCLE_TRIED_TIMES);
        if (cycleTriedTimesObject != null) {
            int cycleTriedTimes = (Integer) cycleTriedTimesObject;
            if (cycleTriedTimes == site.getCycleRetryTimes()) {
               pushErrorSet(request);

            }
        }else if(site.getCycleRetryTimes() == 0){
            pushErrorSet(request);

        }


       // request.getUrl()
    }
    public void pushErrorSet(Request request){
        Jedis jedis = pool.getResource();

        String domain =  UrlUtils.getDomain(request.getUrl());
        String key = ERR_SET_PREFIX+domain;
        jedis.sadd(key,request.getUrl());
        if (request.getExtras() != null) {
            String field = DigestUtils.shaHex(request.getUrl());
            String value = JSON.toJSONString(request);
            jedis.hset((ITEM_PREFIX + domain), field, value);
        }
    }



}
